#include "tokenizer.h"

#include <stddef.h>

#include <stdio.h>

#include <string.h>

#include "fts3_tokenizer.h"

#include "tokenizer/reg.h"

#include "tokenizer/tok.h"

#include "log/log.h"

#include "tags.h"

int tokenizer_init(sqlite3* db) {
	const sqlite3_tokenizer_module* p = tokenizer_nim();
//	log_print(ANDROID_LOG_DEBUG, TAG_NIM_INDEX, "tokenizer_init: ext %p", p);

	int ret = tok_register(db, "nim", p);
//	log_print(ANDROID_LOG_DEBUG, TAG_NIM_INDEX, "tokenizer_init: register %d", ret);

	return ret;
}

void tokenizer_test(const char* str) {
	log_print(ANDROID_LOG_DEBUG, TAG_NIM_INDEX, "tokenizer_test: %s", str);
	const sqlite3_tokenizer_module* pModule = tokenizer_nim();
	if (pModule != NULL) {
		sqlite3_tokenizer *pTokenizer = NULL;
		int ret = pModule->xCreate(0, NULL, &pTokenizer);
		if (pTokenizer != NULL) {
			pTokenizer->pModule = pModule;
			sqlite3_tokenizer_cursor *pCursor = NULL;
			pModule->xOpen(pTokenizer, str, -1, &pCursor);
			if (pCursor != NULL) {
				pCursor->pTokenizer = pTokenizer;
			    const char *pToken = NULL;
			    int nBytes, iStartOffset, iEndOffset, iPosition = 0;
			    while (pModule->xNext(pCursor, &pToken, &nBytes, &iStartOffset, &iEndOffset, &iPosition) == SQLITE_OK) {
			    	char buffer[1024 + 1] = {0};
			    	strncpy(buffer, pToken, nBytes <= 1024 ? nBytes : 1024);
					log_print(ANDROID_LOG_DEBUG, TAG_NIM_INDEX, "tokenizer_test: token %s|%d|%d|%d", buffer, iStartOffset, iEndOffset, iPosition);
				}

				pModule->xClose(pCursor);
			}

			pModule->xDestroy(pTokenizer);
		}
	}
}
